 /*
  *@(#)SpiderDataStream.java		12 Sept. 2000
  *
  * Copyright: MetaPlus Inc & Inquiry Management Systems Ltd.  2000
  *
  * This software is the proprietary information of MetaPlus Inc & 
  * Inquiry Management Systems Ltd.  
  * Use is subject to license terms.
  */
package com.ims.iaa;

import java.io.*; 
import java.net.*; 
import java.sql.*;
import java.rmi.*;
import java.util.LinkedList;
import com.ims.iaa.AD.*;
import com.ims.iaa.Cache.IAppLvl;


/**
  * <p>This class <code>SpiderDataStream</code> is used to support the
  * storing of data collected by a spider during one request search.
  * It either store the data directly to the local file system under the 
  * temp directory or delegate its database store task to server side
  * database service ( the <code>SpiderDBConnection</code> ).</p>
  * <p>Its take two arguments during its construction. The first argument
  * represents the remote request object processed by the spider.
  * The file system or database option is specified with the second 
  * parameter.</p>
  * <p>If data are saved to a file, the file are named as 
  * search site host name followed by search start time with extension ".html",
  * and the sub-directory with the same name contains the image file collected
  * during this search. Image file are simply named as "pic1", "pic2"... with 
  * there repective file extension such as "gif" or "jpeg".</p>
  * 
  * Add the setFlashLink() and creatFlashFile() method by ShengMeng Bai at 2004-07-24
  * Add the sgetAgencyData()() method by ShengMeng Bai at 2004-07-28
  *
  * @author Anthony Zhang
  * @author ShengMeng Bai
  *
  * @version 1.00, 25 Sept. 2000
  * @version 2.00, 24 JUL.  2004
  *
  */
public class SpiderDataStream implements IAppLvl {

	private int imageCount = 0;
	private int m_iFlashCount = 0;
	private int m_iAppletCount = 0;
	private Connection  cn;	
	
	
	private PrintStream ps = null;
	private String fileName  = null;

	private AdTrackRequest adSearch;
	
	private LocalRequest lReq;
	
	private final boolean STOREDB;
	/**
	  * storeInDB indicated wether store to database or not
	  */

	private SpiderDBConnection  sp;
	/**
	 * Use to save the Agency infomation, which is used to
	 * identify whether the advertisement is agency.
	 * this list get through SpiderDBConnection, 
	 * @see getAgencyData
	 */
	private LinkedList m_lstAgency;
	private LinkedList m_lstAdSize;
	/**
	 * Get the agency information from SpiderDBConnection,
	 * the infomation is used to identify whether 
	 * the advertisement is agency.
	 * @author ShengMeng Bai
	 * @return LinkedList which saving the objects of Agency class.
	 */
	public LinkedList getAgencyData(){
		if (m_lstAgency == null){
			if (sp != null){
				try{
					m_lstAgency = sp.loadAgency(IaaThread.getLogString(), lReq.getRequestId());
				} catch (RemoteException e){
					IaaUtil.log("Error contact with the remote Request object " + e);		
				}
			}
		}
		return m_lstAgency;
	}
	//	added by yxy 2008-05-14, for intCriteria calculation	
	public LinkedList getAdSizeData(){
		if(m_lstAdSize == null){
			if(sp != null){
				try {
					m_lstAdSize = sp.loadAdSize(IaaThread.getLogString(),lReq.getRequestId());
				}catch (RemoteException e){
					IaaUtil.log("Error contact with the remote Request object " + e);
				}
			}
		}
		return m_lstAdSize;
	}
	public void setSpiderDB(SpiderDBConnection spCon)  {
		sp = spCon;
	}
	
	public SpiderDataStream(AdTrackRequest adReq,
			boolean storeInDB ) {
		
		adSearch = adReq;
		STOREDB = storeInDB;

		try  {
			lReq = adSearch.getLocalRequest();
		} catch (Exception e) {
			IaaUtil.log("Error contact with the remote Request object " + e);		
		}
				
		if (STOREDB ) {
		} else {
			creatFile();
		}
	}
	
	public void setSearchBegin() {
		//adSearch.setStartTime(
		//	new Timestamp(  System.currentTimeMillis() ) );		
		if (STOREDB ) {

		} else {
			ps.println("<html>");
			ps.print("\r\n");
			try  {
				ps.println("<title>Ad List - "
					+ adSearch.getUrl() +"</title>");
			} catch (Exception e ) {
				IaaUtil.log( "Error get Url as title " + e );
			}
			ps.println("<body>\n");
			ps.println("<table width=100%>");
		}
	}


	public void setSearchEnd() {
		
		try  {
			adSearch.setEndTime(
				new Timestamp(  System.currentTimeMillis() ) );
if(isReachMax() == false)
			adSearch.setSearchState(100);		
else
			adSearch.setSearchState(101);		
		} catch (Exception e ) {
			IaaUtil.log( "Error set End Time " + e );
		}		
					
		if (!STOREDB ) {
			ps.println("</table>");
			ps.println("</body></html>");
			ps.close();
		} 
	}

	public void setSpiderConnection(SpiderConnectionGetter getter) {
		
		SpiderItem si = getter.spiderItem;
		setURLConnection(getter.getURLConnection());
		if (STOREDB ) {
			//store in database
		} else {
			ps.println("<tr bgcolor=#d0d0d0><td>");
			ps.print("<br><b>depth:</b> " + si.depth );
			if ( si.fromUrl!=null)	
				ps.println("&nbsp;&nbsp;<b>context:</b>&nbsp;&nbsp;<a href=\"" + 
					si.fromUrl + "\" target=\"_blank\">" + si.fromUrl+"</a>");
			ps.println("</td></tr>");
			ps.flush(); 
		}
		if (IaaDebug.Debug) 
			IaaUtil.log("Page context: " + si.fromUrl );
	}
	public void setURLConnection(URLConnection uc)  {
		if (STOREDB ) {
			//store in database
		} else {		
			ps.println("<tr bgcolor=#d0d0d0><td>");
				
			ps.print("<b>Page:</b>&nbsp;&nbsp;<a href=\"" + uc.getURL() 
				+ "\" target=\"_blank\">"+ uc.getURL()+"</a>");
			ps.println("</td></tr>");
			ps.flush(); 
		}
		if (IaaDebug.Debug) 
			IaaUtil.log("Visit page " +  uc.getURL());
	}
	
	public void setPageBegin() {
		if (STOREDB ) {

		} else {
			ps.println("<tr><td>");
		}
	}

	public void setPageEnd() {
		if (STOREDB ) {

		} else {
			ps.println("</td></tr>");
		}
	}
	
	//++changsheng 2006-5-16
	public synchronized LocalRequest getLocalRequest()
	{
	    return this.lReq ;
	}
	
	private volatile int impressionsCollected = 0 ;//changsheng
	public synchronized boolean isReachMax(){
	    return impressionsCollected >= lReq.getMaxImpressions() ;
	}
	/**
     * do the fast compare
     * @return
     */
    public synchronized int doFastCompare(ImgLink ad)
    {
    	int intFCResult = FCR_INITAL ;
    	try{
	    	if (sp == null)  {
				//If the spider do not know where to save the data,
				// the request itself should know it.
				sp =lReq.getSpiderDB();
			}
	    	intFCResult = sp.doFastCompare(ad,lReq,IaaThread.getLogString()) ;
if(intFCResult == IAppLvl.FCR_MATCH ){
    impressionsCollected ++ ;
}
    	}
		catch (java.rmi.ConnectException e)  {
			IaaUtil.log(lReq.getSpiderName() + 
				": Connection error " + e);	
			SpecialExceptionLog.getInstance().printStackTrace(e);
			try{
				Thread.sleep(1000);
			}catch (Exception e1){
				
			}
			System.exit(3);
		} catch (RemoteException e)  {
			IaaUtil.log(lReq.getSpiderName() + 
			": Database rollback error " + e);			
		}
    	return intFCResult;
    }
    
    public synchronized int doPreCompare(ImgLink ad)
    {
    	int intFCResult = FCR_INITAL ;
    	try{
	    	if (sp == null)  {
				//If the spider do not know where to save the data,
				// the request itself should know it.
				sp =lReq.getSpiderDB();
			}
	    	intFCResult = sp.doPreCompare(ad,lReq,IaaThread.getLogString()) ;
if(intFCResult == IAppLvl.FCR_MATCH ){
    impressionsCollected ++ ;
}
    	}
		catch (java.rmi.ConnectException e)  {
			IaaUtil.log(lReq.getSpiderName() + 
				": Connection error " + e);	
			SpecialExceptionLog.getInstance().printStackTrace(e);
			try{
				Thread.sleep(1000);
			}catch (Exception e1){
				
			}
			System.exit(3);
		} catch (RemoteException e)  {
			IaaUtil.log(lReq.getSpiderName() + 
			": Database rollback error " + e);			
		}
    	return intFCResult;
    }
    
    /**
     * do really compare
     */
    public synchronized void doCompare(ImgLink ad,int thisImpressionID) 
    {
    	try{
	    	if (sp == null)  {
				//If the spider do not know where to save the data,
				// the request itself should know it.
				sp =lReq.getSpiderDB();
			}
	        sp.doCompare(ad,thisImpressionID,IaaThread.getLogString());
    	} catch (java.rmi.ConnectException e)  {
			IaaUtil.log(lReq.getSpiderName() + 
				": Connection error " + e);	
			SpecialExceptionLog.getInstance().printStackTrace(e);
			try{
				Thread.sleep(1000);
			}catch (Exception e1){
				
			}
			System.exit(3);
		} catch (RemoteException e)  {
			IaaUtil.log(lReq.getSpiderName() + 
			": Database rollback error " + e);			
		}
    }
	//--
	
	//public synchronized void setAdDataLink(AdDataLink adLink){
	public synchronized int setAdDataLink(AdDataLink adLink){
		//++changsheng 2005-5-18
		int thisImpressionID = 0 ;
		//--
		if ( STOREDB ) {
			try  {
				if (sp == null)  {
					//If the spider do not know where to save the data,
					// the request itself should know it.
					sp =lReq.getSpiderDB();
				}
				if(IaaDebug.AuxDebug) IaaUtil.log("Store to databse");
				//++changsheng 2006-5-16
				if(adLink instanceof ImgLink && ((ImgLink)adLink).getIntReferImprID() != 0 ){
					sp.saveCacheAdDataLink((ImgLink)adLink,lReq,IaaThread.getLogString());
				}else{
				//--
					thisImpressionID = sp.storeAdDataLink(adLink,lReq, IaaThread.getLogString());
				}
impressionsCollected ++ ;
			} catch (java.rmi.ConnectException e)  {
				IaaUtil.log(lReq.getSpiderName() + 
					": Connection error " + e);	
				SpecialExceptionLog.getInstance().printStackTrace(e);
				try{
					Thread.sleep(1000);
				}catch (Exception e1){
					
				}
				System.exit(3);
			} catch (RemoteException e)  {
				IaaUtil.log(lReq.getSpiderName() + 
				": Database rollback error " + e);			
			}
		}else{
			if(adLink instanceof ImgLink){
				dispalyImgLink((ImgLink)adLink);
			}else if(adLink instanceof FlashLink){
				dispalyFlashLink((FlashLink)adLink);
			}else if(adLink instanceof AppletLink){
				dispalyAppletLink((AppletLink)adLink);
			}else{
				IaaUtil.log("Fatal error, AdDataLink object is unexcpted type.");
			}
		}
		return thisImpressionID ;//changsheng
	}

	public void println(String x) {
		if (STOREDB ) {
			//store in database
		
		} else {
			ps.println(x);
		}	
	}
	
	private void flush() {
		if (!STOREDB ) {
			ps.flush();
		}			
	}

	public void creatFile()  {
		try {
			String host = URLUtil.getURL(lReq.getUrl()).getHost();
			String time = lReq.getStartTime().toString();
			//fileName = fileName.replace('/', '-');
			time = time.substring(0,19);
			time = time.replace(':', '-'); 
			fileName = host + time;
		} catch (Exception e ) {
			IaaUtil.log( "Cet Url host name failed " + e );
		}
		
		File path = new File("\\temp\\" + fileName );
		try  {
			path.mkdirs();
		} catch (SecurityException e)  {
			IaaUtil.log("Create dir error " + path + e);			
		}
		
		try  {		
			if(IaaDebug.Debug) IaaUtil.log("File name is " + fileName);
									
			ps = new PrintStream( new FileOutputStream( 
						"\\temp\\" + fileName + ".html" ) );
		} catch (Exception e ) {
			IaaUtil.log( "Create file failed " + e );
		}
	}

	/*
	 * Because the ImgLink class has been modify, so this method need modify.
	 * ShengMeng Bai at 2004-07-29
	 */
	private void dispalyImgLink(ImgLink imgLink)  {
		String urlStr = null;
	
		try  {
//			urlStr = URLUtil.getURL(imgLink.containerUrl, imgLink.srcUrlStr).toExternalForm();
			urlStr = URLUtil.getURL(imgLink.getContainerURL(), imgLink.getSrcURLStr()).toExternalForm();
		} catch ( MalformedURLException e ) {
			
		}
	
//		ps.print( "\t<a href=\"" + imgLink.targetUrlStr +"\"  target=\"_blank\">");
		ps.print( "\t<a href=\"" + imgLink.getTargetURLStr() +"\"  target=\"_blank\">");
		ps.println("<img border=1 src=\""+  creatImageFile(imgLink) +"\"></a><br>");
		
		
		//output some information about source
		ps.println("\t<b>source:</b> <a href=\"" + urlStr+
			"\" target=\"_blank\">"+ urlStr +"</a><br>");
//		ps.print("\t<b>width:</b> " + imgLink.width + 
//				"&nbsp;&nbsp;<b>height:</b> " + imgLink.height );
		ps.print("\t<b>width:</b> " + imgLink.getWidth() + 
				"&nbsp;&nbsp;<b>height:</b> " + imgLink.getHeight() );
//		if(imgLink.srcReqStatus==ImgLink.CONN_SUCCESS) {
		if(imgLink.getSrcReqStatus()==ImgLink.CONN_SUCCESS) {
			ps.print("    <b>type:</b> " + imgLink.getAdData().getDataType());
			ps.print("    <b>size:</b> "+ imgLink.getAdData().getDataSize());
		} else {
			ps.print("    <b>type:</b> n/a");
			ps.print("    <b>size:</b> n/a");
		}
		
//		if(imgLink.srcReqStatus!=ImgLink.CONN_SUCCESS) {
		if(imgLink.getSrcReqStatus()!=ImgLink.CONN_SUCCESS) {
			ps.print("\t<b>connect:</b>  " + 
				//ImgLink.statusToString(imgLink.srcReqStatus));
					ImgLink.statusToString(imgLink.getSrcReqStatus()));
		}
		
		ps.println("<br>");
		

		//information about target
/*		ps.print( "\t<b>target:</b> <a href=\""+ imgLink.targetUrlStr +
				"\" target=\"_blank\">"+ imgLink.targetUrlStr +"</a>");
			if (imgLink.realTargetUrlStr!=null)  {
				if (!imgLink.realTargetUrlStr.equals(imgLink.targetUrlStr))  {
					ps.println( "\t<br><b>redirect to :</b><a href=\"" + 
						imgLink.realTargetUrlStr + 	"\" target=\"_blank\">"+ 
						imgLink.realTargetUrlStr+"</a>");
				}
			}
*/
		ps.print( "\t<b>target:</b> <a href=\""+ imgLink.getTargetURLStr() +
				"\" target=\"_blank\">"+ imgLink.getTargetURLStr() +"</a>");
			if (imgLink.getRealTargetURLStr()!=null)  {
				if (!imgLink.getRealTargetURLStr().equals(imgLink.getTargetURLStr()))  {
					ps.println( "\t<br><b>redirect to :</b><a href=\"" + 
						imgLink.getRealTargetURLStr() + 	"\" target=\"_blank\">"+ 
						imgLink.getRealTargetURLStr()+"</a>");
				}
			}
		
/*		if(imgLink.targetReqStatus!=ImgLink.CONN_SUCCESS)  {
			ps.print("\t<b>connect:</b>  " + 
				ImgLink.statusToString(imgLink.targetReqStatus));
		}
*/
		if(imgLink.getTargetReqStatus()!=ImgLink.CONN_SUCCESS)  {
			ps.print("\t<b>connect:</b>  " + 
				ImgLink.statusToString(imgLink.getTargetReqStatus()));
		}
		ps.println("\t<br>");
		//evaluation
		ps.println("\t<b>possibility:</b> "+imgLink.getPossibility()+"%<br>");
		ps.println("\t<b>reason:</b> " + imgLink.getReason() + "<br><br>");
		ps.println("");
		ps.flush();	
	}
		
	private String creatImageFile(ImgLink imgLink)  {
		imageCount++;

		String fileType = imgLink.getAdData().getDataType();
		//remove the phrase "image/"
		fileType = fileType.substring(6,fileType.length());
		
		String imageName = "\\temp\\" + fileName + "\\pic" + 
								imageCount + "." + fileType;
		String imageUrlName = fileName + "\\pic" + imageCount + "." + fileType;
		IaaUtil.log("The image file name is " + imageName);			
		
		InputStream in = imgLink.getAdData().getData();
		if( in != null)  {
			try  {
				BufferedOutputStream out = new BufferedOutputStream( 
					new FileOutputStream(imageName) );

				byte[] buf = new byte[1024];
				int readNum =0;

				while ( (readNum = in.read(buf)) != -1)  {
					out.write(buf,0,readNum);
				}
				out.close();
				in.close();
			} catch (IOException e)  {
				IaaUtil.log("Error create image file " + e);
			}
		}
		return imageUrlName;
	}

	/**
	 * write the flash infomation to the html file.
	 * @param flashLink Flash link data
	 * @author ShengMeng Bai
	 */
	private void dispalyFlashLink( FlashLink flashLink)  {
	
		String urlStr = null;
		
		try  {
			urlStr = URLUtil.getURL( flashLink.getContainerURL(),
					flashLink.getSrcURL().toExternalForm() ).toExternalForm();
		} catch ( MalformedURLException e ) {
			
		}
		ps.println( "<b>Find Flash:    <b>"); 
		ps.println( "\t<object classid=\"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000\" codebase=\""
				+ flashLink.getCodeBase() + "\""
				+ " width=" + flashLink.getWidth()
				+ " height=" + flashLink.getHeight()
				+ ">");
		ps.println( "\t\t<param name=movie value=\""
				+ creatFlashFile( flashLink )
				+ "\">" );
		ps.println( "\t</object>" );
		ps.println( "<br>" );
		ps.println( "<b>Source:</b> <a href=\""
				+ flashLink.getSrcURL().toExternalForm()
				+ "\" target=\"_blank\">"+ urlStr +"</a><br>");
		ps.print("\t<b>width:</b> " + flashLink.getWidth() + 
				"&nbsp;&nbsp;<b>height:</b> " + flashLink.getHeight() );
		if( flashLink.getSrcReqStatus() == FlashLink.CONN_SUCCESS ) {
			ps.print("    <b>size:</b> "+ flashLink.getAdData().getDataSize() );
		} else {
			ps.print("    <b>size:</b> n/a");
		}
		ps.println( "<br>" );
		ps.flush();	
	}
	/**
	 * Create the flash file
	 * @param flashLink Flash link data
	 * @return the created file name which contain the relative path.
	 * @author ShengMeng Bai
	 */
	private String creatFlashFile( FlashLink flashLink )  {
		m_iFlashCount++;

		String sFlashName = "\\temp\\" + fileName + "\\flash" + 
			m_iFlashCount + ".swf" ;
		String sFlashUrlName = fileName + "\\flash" + m_iFlashCount + ".swf";
		IaaUtil.log("The flash file name is " + sFlashName);			
		
		InputStream in = flashLink.getAdData().getData();
		if( in != null)  {
			try  {
				BufferedOutputStream out = new BufferedOutputStream( 
					new FileOutputStream(sFlashName) );

				byte[] buf = new byte[1024];
				int readNum =0;

				while ( (readNum = in.read(buf)) != -1)  {
					out.write(buf,0,readNum);
				}
				out.close();
				in.close();
			} catch (IOException e)  {
				IaaUtil.log("Error create flash file " + e);
			}
		}
		return sFlashUrlName;
	}

	/**
	 * write the flash infomation to the html file.
	 * @param flashLink Flash link data
	 * @author ShengMeng Bai
	 */
	private void dispalyAppletLink( AppletLink appletLink)  {
	
		String urlStr = null;
		
		try  {
			urlStr = URLUtil.getURL( appletLink.getContainerURL(),
					appletLink.getSrcURL().toExternalForm() ).toExternalForm();
		} catch ( MalformedURLException e ) {
			
		}
		ps.println( "<b>Find Applet:    <b>");
		String sJar = creatAppletFile( appletLink );
		String sApplet = appletLink.getAppletParamStr();
		int iStart = sApplet.indexOf("archive=\"");
		if(iStart != -1){
			int iEnd = sApplet.indexOf(" ", iStart);
			sApplet = sApplet.substring(0, iStart) + "archive=\"" + sJar + "\"" + sApplet.substring(iEnd);
			iStart = sApplet.indexOf("codebase=\"");
			iEnd = sApplet.indexOf(" ", iStart);
			sApplet = sApplet.substring(0, iStart) + sApplet.substring(iEnd);
		}else{
			
		}
		ps.println( sApplet );
		ps.println( "<br>" );
		ps.print("\t<b>width:</b> " + appletLink.getWidth() + 
				"&nbsp;&nbsp;<b>height:</b> " + appletLink.getHeight() );
		if( appletLink.getSrcReqStatus() == FlashLink.CONN_SUCCESS ) {
			ps.print("    <b>size:</b> "+ appletLink.getAdData().getDataSize() );
		} else {
			ps.print("    <b>size:</b> n/a");
		}
		ps.println( "<br>" );
		ps.flush();	
	}
	/**
	 * Create the flash file
	 * @param flashLink Flash link data
	 * @return the created file name which contain the relative path.
	 * @author ShengMeng Bai
	 */
	private String creatAppletFile( AppletLink appletLink )  {
		m_iAppletCount++;

		String sJarName = "\\temp\\" + fileName + "\\jar" + 
		m_iAppletCount + ".jar" ;
		String sJarUrlName = fileName + "\\jar" + m_iAppletCount + ".jar";
		IaaUtil.log("The jar file name is " + sJarName);			
		
		InputStream in = appletLink.getAdData().getData();
		if( in != null)  {
			try  {
				BufferedOutputStream out = new BufferedOutputStream( 
					new FileOutputStream(sJarName) );

				byte[] buf = new byte[1024];
				int readNum =0;

				while ( (readNum = in.read(buf)) != -1)  {
					out.write(buf,0,readNum);
				}
				out.close();
				in.close();
			} catch (IOException e)  {
				IaaUtil.log("Error create jar file " + e);
			}
		}
		return sJarUrlName;
	}
}
	
